{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation.evaluate import evaluate\n",
    "from data_utils.basic_data_round2 import load_train_val_dataset,load_basic_dataset\n",
    "from results_process.regulizer import remove_nine,remove_short_entity\n",
    "from results_process.utils import load_model_rs\n",
    "from results_process.bert_entity_model import reduce_rs_by_id\n",
    "from functools import reduce\n",
    "import numpy as np\n",
    "\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "from glob import glob\n",
    "from os.path import join\n",
    "MODEL_ROOT = '/home/njuciairs/wangshuai/competitions/finacial_models_round2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/home/njuciairs/wangshuai/competitions/finacial_models_round2\u001b[00m\r\n",
      "├── \u001b[01;34mBertSentiEntity_cross\u001b[00m\r\n",
      "├── \u001b[01;34mbinary_class_cross_roberta\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573410905.AIRS.24109.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561033.AIRS.23888.12\r\n",
      "│   │   │       ├── events.out.tfevents.1573561463.AIRS.23888.21\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573410910.AIRS.24109.1\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573414365.AIRS.24109.4\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573414365.AIRS.24109.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573412612.AIRS.24109.2\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573446913.AIRS.7323.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561084.AIRS.23888.13\r\n",
      "│   │   │       ├── events.out.tfevents.1573561865.AIRS.23888.22\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573446917.AIRS.7323.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573448534.AIRS.7323.2\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573467021.AIRS.26865.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561134.AIRS.23888.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573562266.AIRS.23888.23\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573467025.AIRS.26865.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573468955.AIRS.26865.2\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573478138.AIRS.26865.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573561183.AIRS.23888.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573562668.AIRS.23888.24\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573478140.AIRS.26865.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573479810.AIRS.26865.5\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573490017.AIRS.26865.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573561234.AIRS.23888.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573563069.AIRS.23888.25\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573490019.AIRS.26865.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573491923.AIRS.26865.8\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573526921.AIRS.23888.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573561280.AIRS.23888.17\r\n",
      "│   │   │       ├── events.out.tfevents.1573563470.AIRS.23888.26\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573526925.AIRS.23888.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573528553.AIRS.23888.2\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573537414.AIRS.23888.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573561325.AIRS.23888.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573563871.AIRS.23888.27\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573537417.AIRS.23888.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573538960.AIRS.23888.5\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573543886.AIRS.23888.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573561371.AIRS.23888.19\r\n",
      "│   │   │       ├── events.out.tfevents.1573564273.AIRS.23888.28\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573543888.AIRS.23888.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573545363.AIRS.23888.8\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573553232.AIRS.23888.9\r\n",
      "│   │           ├── events.out.tfevents.1573561415.AIRS.23888.20\r\n",
      "│   │           ├── events.out.tfevents.1573564674.AIRS.23888.29\r\n",
      "│   │           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573553234.AIRS.23888.10\r\n",
      "│   │           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573554689.AIRS.23888.11\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.979913_epoch7_time20191111054906\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.972230_epoch7_time20191111154518\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.964897_epoch5_time20191111203714\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.976054_epoch3_time20191111224656\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.976327_epoch3_time20191112020819\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.980947_epoch5_time20191112130505\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.977693_epoch1_time20191112140920\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.970953_epoch5_time20191112173350\r\n",
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.974504_epoch2_time20191112185543\r\n",
      "├── \u001b[01;34mmulti_class_cross1\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573026942.AIRS.13972.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573196075.AIRS.15210.27\r\n",
      "│   │   │       ├── events.out.tfevents.1573398630.AIRS.15463.0\r\n",
      "│   │   │       └── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573026950.AIRS.13972.1\r\n",
      "│   │   ├── \u001b[01;34m10\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573180302.AIRS.15210.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573399967.AIRS.22163.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573412781.AIRS.28509.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573180305.AIRS.15210.25\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573181396.AIRS.15210.26\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573027124.AIRS.15210.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573398766.AIRS.16498.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573399702.AIRS.22163.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573400088.AIRS.22163.9\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573027132.AIRS.15210.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573028203.AIRS.15210.2\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573043806.AIRS.15210.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573398804.AIRS.16498.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573399735.AIRS.22163.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573400605.AIRS.22163.10\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573043812.AIRS.15210.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573044886.AIRS.15210.5\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573057365.AIRS.15210.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573398863.AIRS.16498.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573399770.AIRS.22163.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573401041.AIRS.22163.11\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573057369.AIRS.15210.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573058449.AIRS.15210.8\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573086938.AIRS.15210.9\r\n",
      "│   │   │       ├── events.out.tfevents.1573398960.AIRS.16498.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573399802.AIRS.22163.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573402451.AIRS.22163.12\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573086942.AIRS.15210.10\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573088023.AIRS.15210.11\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573100391.AIRS.15210.12\r\n",
      "│   │   │       ├── events.out.tfevents.1573398996.AIRS.16498.4\r\n",
      "│   │   │       ├── events.out.tfevents.1573399838.AIRS.22163.4\r\n",
      "│   │   │       ├── events.out.tfevents.1573402819.AIRS.22163.13\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573100395.AIRS.15210.13\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573101479.AIRS.15210.14\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573118523.AIRS.15210.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573399137.AIRS.16498.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573399869.AIRS.22163.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573403212.AIRS.22163.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573411602.AIRS.28086.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573411678.AIRS.28509.0\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573118530.AIRS.15210.16\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573119596.AIRS.15210.17\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573135028.AIRS.15210.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573399188.AIRS.16498.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573399902.AIRS.22163.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573412046.AIRS.28509.1\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573135031.AIRS.15210.19\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573136118.AIRS.15210.20\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573157183.AIRS.15210.21\r\n",
      "│   │           ├── events.out.tfevents.1573399226.AIRS.16498.7\r\n",
      "│   │           ├── events.out.tfevents.1573399932.AIRS.22163.7\r\n",
      "│   │           ├── events.out.tfevents.1573412413.AIRS.28509.2\r\n",
      "│   │           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573157188.AIRS.15210.22\r\n",
      "│   │           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573158267.AIRS.15210.23\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mdata\u001b[00m\r\n",
      "│   ├── \u001b[01;34mversion10\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.918750_epoch4_time20191108114352\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.908206_epoch5_time20191106172749\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.921625_epoch2_time20191106211303\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.902092_epoch7_time20191107022822\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.928776_epoch2_time20191107091135\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.920174_epoch6_time20191107140826\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.923510_epoch5_time20191107185117\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.933245_epoch9_time20191108003930\r\n",
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.916217_epoch3_time20191108050017\r\n",
      "├── \u001b[01;34mmulti_class_cross_roberta\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573408955.AIRS.12503.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573534379.AIRS.27396.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573534668.AIRS.27396.27\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573408962.AIRS.12503.1\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573411125.AIRS.12503.4\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573411125.AIRS.12503.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573410044.AIRS.12503.2\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573424990.AIRS.8891.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573534413.AIRS.27396.19\r\n",
      "│   │   │       ├── events.out.tfevents.1573535035.AIRS.27396.28\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573424997.AIRS.8891.1\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573427150.AIRS.8891.4\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573427150.AIRS.8891.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573426074.AIRS.8891.2\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573438279.AIRS.8891.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573534447.AIRS.27396.20\r\n",
      "│   │   │       ├── events.out.tfevents.1573535400.AIRS.27396.29\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573438282.AIRS.8891.6\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573440444.AIRS.8891.9\r\n",
      "│   │   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573440444.AIRS.8891.8\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573439362.AIRS.8891.7\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573461523.AIRS.27396.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573534481.AIRS.27396.21\r\n",
      "│   │   │       ├── events.out.tfevents.1573535766.AIRS.27396.30\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461530.AIRS.27396.1\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573462629.AIRS.27396.2\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573475114.AIRS.27396.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573534515.AIRS.27396.22\r\n",
      "│   │   │       ├── events.out.tfevents.1573536131.AIRS.27396.31\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573475118.AIRS.27396.4\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573476202.AIRS.27396.5\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573487672.AIRS.27396.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573534548.AIRS.27396.23\r\n",
      "│   │   │       ├── events.out.tfevents.1573536497.AIRS.27396.32\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573487675.AIRS.27396.7\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573488770.AIRS.27396.8\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573500215.AIRS.27396.9\r\n",
      "│   │   │       ├── events.out.tfevents.1573534579.AIRS.27396.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573536862.AIRS.27396.33\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573500218.AIRS.27396.10\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573501323.AIRS.27396.11\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573511761.AIRS.27396.12\r\n",
      "│   │   │       ├── events.out.tfevents.1573534608.AIRS.27396.25\r\n",
      "│   │   │       ├── events.out.tfevents.1573537228.AIRS.27396.34\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573511764.AIRS.27396.13\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573512864.AIRS.27396.14\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573519854.AIRS.27396.15\r\n",
      "│   │           ├── events.out.tfevents.1573534638.AIRS.27396.26\r\n",
      "│   │           ├── events.out.tfevents.1573537594.AIRS.27396.35\r\n",
      "│   │           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573519857.AIRS.27396.16\r\n",
      "│   │           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573520937.AIRS.27396.17\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.922166_epoch3_time20191111030235\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.926959_epoch6_time20191111082927\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.915481_epoch5_time20191111115139\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.936144_epoch7_time20191111184620\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.917137_epoch6_time20191111221356\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.921900_epoch6_time20191112014343\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.932367_epoch5_time20191112045514\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.907922_epoch2_time20191112071221\r\n",
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.919583_epoch8_time20191112111410\r\n",
      "├── \u001b[01;34msenti_entity_goodremove\u001b[00m\r\n",
      "├── \u001b[01;34msenti_entity_goodremove_full\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573461283.AIRS.25939.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573470493.AIRS.25939.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573479455.AIRS.1625.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573479466.AIRS.1625.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573487448.AIRS.1625.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573495423.AIRS.1625.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573502486.AIRS.1625.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573511739.AIRS.1625.32\r\n",
      "│   │   │       ├── events.out.tfevents.1573511923.AIRS.1625.34\r\n",
      "│   │   │       ├── events.out.tfevents.1573512024.AIRS.1625.39\r\n",
      "│   │   │       ├── events.out.tfevents.1573544153.AIRS.27176.0\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461292.AIRS.25939.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461292.AIRS.25939.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461944.AIRS.25939.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573461944.AIRS.25939.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573461944.AIRS.25939.5\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573479653.AIRS.1625.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573487454.AIRS.1625.9\r\n",
      "│   │   │       ├── events.out.tfevents.1573511948.AIRS.1625.35\r\n",
      "│   │   │       ├── events.out.tfevents.1573512194.AIRS.1625.40\r\n",
      "│   │   │       ├── events.out.tfevents.1573544208.AIRS.27176.1\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573479660.AIRS.1625.4\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573479660.AIRS.1625.3\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573480313.AIRS.1625.6\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573480313.AIRS.1625.5\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573480313.AIRS.1625.7\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573487632.AIRS.1625.10\r\n",
      "│   │   │       ├── events.out.tfevents.1573495428.AIRS.1625.17\r\n",
      "│   │   │       ├── events.out.tfevents.1573511973.AIRS.1625.36\r\n",
      "│   │   │       ├── events.out.tfevents.1573512364.AIRS.1625.41\r\n",
      "│   │   │       ├── events.out.tfevents.1573544261.AIRS.27176.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573487639.AIRS.1625.12\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573487639.AIRS.1625.11\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573488292.AIRS.1625.14\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573488292.AIRS.1625.13\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573488292.AIRS.1625.15\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573495862.AIRS.1625.18\r\n",
      "│   │   │       ├── events.out.tfevents.1573502491.AIRS.1625.25\r\n",
      "│   │   │       ├── events.out.tfevents.1573511990.AIRS.1625.37\r\n",
      "│   │   │       ├── events.out.tfevents.1573512534.AIRS.1625.42\r\n",
      "│   │   │       ├── events.out.tfevents.1573544316.AIRS.27176.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573495996.AIRS.1625.20\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573495996.AIRS.1625.19\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573496649.AIRS.1625.22\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573496649.AIRS.1625.21\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573496649.AIRS.1625.23\r\n",
      "│   │   └── \u001b[01;34m5\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573502668.AIRS.1625.26\r\n",
      "│   │           ├── events.out.tfevents.1573511745.AIRS.1625.33\r\n",
      "│   │           ├── events.out.tfevents.1573512007.AIRS.1625.38\r\n",
      "│   │           ├── events.out.tfevents.1573512704.AIRS.1625.43\r\n",
      "│   │           ├── events.out.tfevents.1573544399.AIRS.27176.4\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573502674.AIRS.1625.28\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573502674.AIRS.1625.27\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573503328.AIRS.1625.30\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573503328.AIRS.1625.29\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573503328.AIRS.1625.31\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.963447_epoch8_time20191111180125\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.962730_epoch8_time20191111230740\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.962609_epoch8_time20191112012036\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.961126_epoch6_time20191112031816\r\n",
      "│   └── \u001b[01;34mversion5\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.961709_epoch8_time20191112053105\r\n",
      "├── \u001b[01;34msenti_entity_goodremove_on_round1data\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573213491.AIRS.30894.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573219507.AIRS.30894.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573219518.AIRS.30894.7\r\n",
      "│   │   │       ├── events.out.tfevents.1573224890.AIRS.30894.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573233218.AIRS.30894.22\r\n",
      "│   │   │       ├── events.out.tfevents.1573239879.AIRS.30894.30\r\n",
      "│   │   │       ├── events.out.tfevents.1573247885.AIRS.30894.38\r\n",
      "│   │   │       ├── events.out.tfevents.1573255073.AIRS.30894.46\r\n",
      "│   │   │       ├── events.out.tfevents.1573259572.AIRS.30894.54\r\n",
      "│   │   │       ├── events.out.tfevents.1573266749.AIRS.30894.62\r\n",
      "│   │   │       ├── events.out.tfevents.1573272065.AIRS.30894.70\r\n",
      "│   │   │       ├── events.out.tfevents.1573272254.AIRS.30894.72\r\n",
      "│   │   │       ├── events.out.tfevents.1573398902.AIRS.17318.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573399771.AIRS.22565.0\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213505.AIRS.30894.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213505.AIRS.30894.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213778.AIRS.30894.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573213778.AIRS.30894.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573213778.AIRS.30894.5\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573219700.AIRS.30894.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573224902.AIRS.30894.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573272277.AIRS.30894.73\r\n",
      "│   │   │       ├── events.out.tfevents.1573399010.AIRS.17318.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573399951.AIRS.22565.1\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219704.AIRS.30894.10\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219704.AIRS.30894.9\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219980.AIRS.30894.12\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573219980.AIRS.30894.11\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573219980.AIRS.30894.13\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573225083.AIRS.30894.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573233222.AIRS.30894.23\r\n",
      "│   │   │       ├── events.out.tfevents.1573272299.AIRS.30894.74\r\n",
      "│   │   │       ├── events.out.tfevents.1573399088.AIRS.17318.2\r\n",
      "│   │   │       ├── events.out.tfevents.1573400128.AIRS.22565.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225088.AIRS.30894.18\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225088.AIRS.30894.17\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225364.AIRS.30894.20\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573225364.AIRS.30894.19\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573225364.AIRS.30894.21\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573233398.AIRS.30894.24\r\n",
      "│   │   │       ├── events.out.tfevents.1573239884.AIRS.30894.31\r\n",
      "│   │   │       ├── events.out.tfevents.1573272322.AIRS.30894.75\r\n",
      "│   │   │       ├── events.out.tfevents.1573399140.AIRS.17318.3\r\n",
      "│   │   │       ├── events.out.tfevents.1573400303.AIRS.22565.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233402.AIRS.30894.26\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233402.AIRS.30894.25\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233675.AIRS.30894.28\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573233675.AIRS.30894.27\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573233675.AIRS.30894.29\r\n",
      "│   │   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573240060.AIRS.30894.32\r\n",
      "│   │   │       ├── events.out.tfevents.1573247889.AIRS.30894.39\r\n",
      "│   │   │       ├── events.out.tfevents.1573272345.AIRS.30894.76\r\n",
      "│   │   │       ├── events.out.tfevents.1573399428.AIRS.17318.4\r\n",
      "│   │   │       ├── events.out.tfevents.1573400478.AIRS.22565.4\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240064.AIRS.30894.34\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240064.AIRS.30894.33\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240336.AIRS.30894.36\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573240336.AIRS.30894.35\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573240336.AIRS.30894.37\r\n",
      "│   │   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573248064.AIRS.30894.40\r\n",
      "│   │   │       ├── events.out.tfevents.1573255076.AIRS.30894.47\r\n",
      "│   │   │       ├── events.out.tfevents.1573272368.AIRS.30894.77\r\n",
      "│   │   │       ├── events.out.tfevents.1573399478.AIRS.17318.5\r\n",
      "│   │   │       ├── events.out.tfevents.1573400664.AIRS.22565.5\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248068.AIRS.30894.42\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248068.AIRS.30894.41\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248341.AIRS.30894.44\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573248341.AIRS.30894.43\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573248341.AIRS.30894.45\r\n",
      "│   │   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573255252.AIRS.30894.48\r\n",
      "│   │   │       ├── events.out.tfevents.1573259575.AIRS.30894.55\r\n",
      "│   │   │       ├── events.out.tfevents.1573272383.AIRS.30894.78\r\n",
      "│   │   │       ├── events.out.tfevents.1573399507.AIRS.17318.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573401300.AIRS.22565.6\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255256.AIRS.30894.50\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255256.AIRS.30894.49\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255528.AIRS.30894.52\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573255528.AIRS.30894.51\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573255528.AIRS.30894.53\r\n",
      "│   │   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573259751.AIRS.30894.56\r\n",
      "│   │   │       ├── events.out.tfevents.1573266753.AIRS.30894.63\r\n",
      "│   │   │       ├── events.out.tfevents.1573272397.AIRS.30894.79\r\n",
      "│   │   │       ├── events.out.tfevents.1573399531.AIRS.17318.7\r\n",
      "│   │   │       ├── events.out.tfevents.1573401485.AIRS.22565.7\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573259756.AIRS.30894.58\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573259755.AIRS.30894.57\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573260028.AIRS.30894.60\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573260028.AIRS.30894.59\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573260028.AIRS.30894.61\r\n",
      "│   │   └── \u001b[01;34m9\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573266930.AIRS.30894.64\r\n",
      "│   │           ├── events.out.tfevents.1573272069.AIRS.30894.71\r\n",
      "│   │           ├── events.out.tfevents.1573272411.AIRS.30894.80\r\n",
      "│   │           ├── events.out.tfevents.1573399554.AIRS.17318.8\r\n",
      "│   │           ├── events.out.tfevents.1573401681.AIRS.22565.8\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573266936.AIRS.30894.66\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573266936.AIRS.30894.65\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573267208.AIRS.30894.68\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573267208.AIRS.30894.67\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573267208.AIRS.30894.69\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.935708_epoch5_time20191108200756\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.965252_epoch9_time20191108220927\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.951920_epoch9_time20191108233913\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.952887_epoch5_time20191109013920\r\n",
      "│   ├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.949487_epoch6_time20191109033451\r\n",
      "│   ├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.943984_epoch4_time20191109053914\r\n",
      "│   ├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.953871_epoch6_time20191109074801\r\n",
      "│   ├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.925111_epoch6_time20191109090301\r\n",
      "│   └── \u001b[01;34mversion9\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       └── score0.955598_epoch9_time20191109111611\r\n",
      "├── \u001b[01;34msenti_entity_inplace_full\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   └── \u001b[01;34m1\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573711871.AIRS.6990.0\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573711877.AIRS.6990.2\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573711877.AIRS.6990.1\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573712531.AIRS.6990.4\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573712531.AIRS.6990.3\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573712531.AIRS.6990.5\r\n",
      "│   └── \u001b[01;34mversion1\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       └── score0.962004_epoch7_time20191114152737\r\n",
      "├── \u001b[01;34msenti_entity_inplace_full_v1\u001b[00m\r\n",
      "│   ├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573727231.AIRS.1640.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573734958.AIRS.14407.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573734967.AIRS.14407.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573735142.AIRS.16060.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573735162.AIRS.16060.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573801084.AIRS.8283.6\r\n",
      "│   │   │       ├── events.out.tfevents.1573810723.AIRS.8283.14\r\n",
      "│   │   │       ├── events.out.tfevents.1573818260.AIRS.8283.22\r\n",
      "│   │   │       ├── events.out.tfevents.1573828433.AIRS.8283.30\r\n",
      "│   │   │       ├── events.out.tfevents.1573828644.AIRS.8283.32\r\n",
      "│   │   │       ├── events.out.tfevents.1573884890.AIRS.21602.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573885011.AIRS.21741.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573885718.AIRS.21741.5\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727236.AIRS.1640.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727236.AIRS.1640.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727916.AIRS.1640.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573727916.AIRS.1640.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573727916.AIRS.1640.5\r\n",
      "│   │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573791715.AIRS.8283.0\r\n",
      "│   │   │       ├── events.out.tfevents.1573801229.AIRS.8283.7\r\n",
      "│   │   │       ├── events.out.tfevents.1573885099.AIRS.21741.1\r\n",
      "│   │   │       ├── events.out.tfevents.1573886317.AIRS.21741.6\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573791729.AIRS.8283.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573791729.AIRS.8283.1\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573792397.AIRS.8283.4\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573792397.AIRS.8283.3\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573792397.AIRS.8283.5\r\n",
      "│   │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573801413.AIRS.8283.8\r\n",
      "│   │   │       ├── events.out.tfevents.1573810731.AIRS.8283.15\r\n",
      "│   │   │       ├── events.out.tfevents.1573885194.AIRS.21741.2\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573801421.AIRS.8283.10\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573801421.AIRS.8283.9\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573802091.AIRS.8283.12\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573802091.AIRS.8283.11\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573802091.AIRS.8283.13\r\n",
      "│   │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │   │       ├── events.out.tfevents.1573810914.AIRS.8283.16\r\n",
      "│   │   │       ├── events.out.tfevents.1573818293.AIRS.8283.23\r\n",
      "│   │   │       ├── events.out.tfevents.1573885314.AIRS.21741.3\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573810935.AIRS.8283.18\r\n",
      "│   │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573810935.AIRS.8283.17\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573811601.AIRS.8283.20\r\n",
      "│   │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │   │       │   └── events.out.tfevents.1573811601.AIRS.8283.19\r\n",
      "│   │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │   │           └── events.out.tfevents.1573811601.AIRS.8283.21\r\n",
      "│   │   └── \u001b[01;34m5\u001b[00m\r\n",
      "│   │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │           ├── events.out.tfevents.1573818492.AIRS.8283.24\r\n",
      "│   │           ├── events.out.tfevents.1573828443.AIRS.8283.31\r\n",
      "│   │           ├── events.out.tfevents.1573885555.AIRS.21741.4\r\n",
      "│   │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573818506.AIRS.8283.26\r\n",
      "│   │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573818506.AIRS.8283.25\r\n",
      "│   │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573819175.AIRS.8283.28\r\n",
      "│   │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │           │   └── events.out.tfevents.1573819175.AIRS.8283.27\r\n",
      "│   │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │               └── events.out.tfevents.1573819175.AIRS.8283.29\r\n",
      "│   ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   └── score0.967381_epoch7_time20191114194541\r\n",
      "│   ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   ├── score0.951676_epoch1_time20191115123317\r\n",
      "│   │   ├── score0.956579_epoch2_time20191115124426\r\n",
      "│   │   ├── score0.960477_epoch3_time20191115125534\r\n",
      "│   │   ├── score0.963445_epoch4_time20191115130640\r\n",
      "│   │   ├── score0.964204_epoch5_time20191115131746\r\n",
      "│   │   ├── score0.965986_epoch7_time20191115133956\r\n",
      "│   │   ├── score0.968415_epoch8_time20191115135102\r\n",
      "│   │   └── score0.970270_epoch10_time20191115141311\r\n",
      "│   ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   ├── score0.943036_epoch1_time20191115151451\r\n",
      "│   │   ├── score0.958119_epoch2_time20191115152601\r\n",
      "│   │   ├── score0.958387_epoch3_time20191115153709\r\n",
      "│   │   ├── score0.961559_epoch4_time20191115154816\r\n",
      "│   │   ├── score0.961809_epoch5_time20191115155923\r\n",
      "│   │   ├── score0.963421_epoch7_time20191115162131\r\n",
      "│   │   ├── score0.963601_epoch9_time20191115164335\r\n",
      "│   │   └── score0.966205_epoch10_time20191115165438\r\n",
      "│   ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   │   └── raw_rs.csv\r\n",
      "│   │   ├── score0.957020_epoch1_time20191115175321\r\n",
      "│   │   ├── score0.959496_epoch2_time20191115180427\r\n",
      "│   │   ├── score0.967964_epoch3_time20191115181532\r\n",
      "│   │   └── score0.969034_epoch7_time20191115185937\r\n",
      "│   └── \u001b[01;34mversion5\u001b[00m\r\n",
      "│       ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│       │   └── raw_rs.csv\r\n",
      "│       ├── score0.951851_epoch1_time20191115195935\r\n",
      "│       ├── score0.959211_epoch2_time20191115201041\r\n",
      "│       ├── score0.959282_epoch3_time20191115202146\r\n",
      "│       ├── score0.964579_epoch4_time20191115203250\r\n",
      "│       ├── score0.964818_epoch5_time20191115204353\r\n",
      "│       ├── score0.966388_epoch8_time20191115211656\r\n",
      "│       └── score0.966862_epoch11_time20191115214954\r\n",
      "└── \u001b[01;34msenti_entity_inplace_hit_full_v1\u001b[00m\r\n",
      "    ├── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   ├── \u001b[01;34m1\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573791469.AIRS.7248.0\r\n",
      "    │   │       ├── events.out.tfevents.1573801168.AIRS.7248.6\r\n",
      "    │   │       ├── events.out.tfevents.1573801176.AIRS.7248.7\r\n",
      "    │   │       ├── events.out.tfevents.1573811018.AIRS.7248.14\r\n",
      "    │   │       ├── events.out.tfevents.1573820841.AIRS.7248.22\r\n",
      "    │   │       ├── events.out.tfevents.1573830661.AIRS.7248.30\r\n",
      "    │   │       ├── events.out.tfevents.1573840552.AIRS.7248.38\r\n",
      "    │   │       ├── events.out.tfevents.1573840724.AIRS.7248.40\r\n",
      "    │   │       ├── events.out.tfevents.1573885031.AIRS.22007.0\r\n",
      "    │   │       ├── events.out.tfevents.1573885435.AIRS.22007.5\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573791475.AIRS.7248.2\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573791475.AIRS.7248.1\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573792125.AIRS.7248.4\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573792125.AIRS.7248.3\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573792125.AIRS.7248.5\r\n",
      "    │   ├── \u001b[01;34m2\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573801350.AIRS.7248.8\r\n",
      "    │   │       ├── events.out.tfevents.1573811021.AIRS.7248.15\r\n",
      "    │   │       ├── events.out.tfevents.1573885117.AIRS.22007.1\r\n",
      "    │   │       ├── events.out.tfevents.1573885740.AIRS.22007.6\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573801353.AIRS.7248.10\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573801353.AIRS.7248.9\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573802005.AIRS.7248.12\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573802005.AIRS.7248.11\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573802005.AIRS.7248.13\r\n",
      "    │   ├── \u001b[01;34m3\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573811191.AIRS.7248.16\r\n",
      "    │   │       ├── events.out.tfevents.1573820844.AIRS.7248.23\r\n",
      "    │   │       ├── events.out.tfevents.1573885208.AIRS.22007.2\r\n",
      "    │   │       ├── events.out.tfevents.1573885913.AIRS.22007.7\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811194.AIRS.7248.18\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811194.AIRS.7248.17\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811842.AIRS.7248.20\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573811842.AIRS.7248.19\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573811842.AIRS.7248.21\r\n",
      "    │   ├── \u001b[01;34m4\u001b[00m\r\n",
      "    │   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │   │       ├── events.out.tfevents.1573821014.AIRS.7248.24\r\n",
      "    │   │       ├── events.out.tfevents.1573830664.AIRS.7248.31\r\n",
      "    │   │       ├── events.out.tfevents.1573885326.AIRS.22007.3\r\n",
      "    │   │       ├── events.out.tfevents.1573886207.AIRS.22007.8\r\n",
      "    │   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821017.AIRS.7248.26\r\n",
      "    │   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821017.AIRS.7248.25\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821665.AIRS.7248.28\r\n",
      "    │   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │   │       │   └── events.out.tfevents.1573821665.AIRS.7248.27\r\n",
      "    │   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │   │           └── events.out.tfevents.1573821665.AIRS.7248.29\r\n",
      "    │   └── \u001b[01;34m5\u001b[00m\r\n",
      "    │       └── \u001b[01;34mboard\u001b[00m\r\n",
      "    │           ├── events.out.tfevents.1573830834.AIRS.7248.32\r\n",
      "    │           ├── events.out.tfevents.1573840554.AIRS.7248.39\r\n",
      "    │           ├── events.out.tfevents.1573885376.AIRS.22007.4\r\n",
      "    │           ├── events.out.tfevents.1573886549.AIRS.22007.9\r\n",
      "    │           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573830837.AIRS.7248.34\r\n",
      "    │           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573830837.AIRS.7248.33\r\n",
      "    │           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573831488.AIRS.7248.36\r\n",
      "    │           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "    │           │   └── events.out.tfevents.1573831488.AIRS.7248.35\r\n",
      "    │           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "    │               └── events.out.tfevents.1573831488.AIRS.7248.37\r\n",
      "    ├── \u001b[01;34mversion1\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.929784_epoch1_time20191115122845\r\n",
      "    │   ├── score0.942125_epoch2_time20191115123935\r\n",
      "    │   ├── score0.945590_epoch3_time20191115125024\r\n",
      "    │   ├── score0.952973_epoch4_time20191115130111\r\n",
      "    │   ├── score0.955620_epoch6_time20191115132242\r\n",
      "    │   ├── score0.955754_epoch7_time20191115133327\r\n",
      "    │   ├── score0.957575_epoch8_time20191115134413\r\n",
      "    │   ├── score0.958603_epoch9_time20191115135459\r\n",
      "    │   ├── score0.960612_epoch10_time20191115140544\r\n",
      "    │   ├── score0.960965_epoch11_time20191115141628\r\n",
      "    │   └── score0.963365_epoch12_time20191115142713\r\n",
      "    ├── \u001b[01;34mversion2\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.917796_epoch1_time20191115151325\r\n",
      "    │   ├── score0.938023_epoch2_time20191115152415\r\n",
      "    │   ├── score0.947443_epoch4_time20191115154550\r\n",
      "    │   ├── score0.949963_epoch5_time20191115155638\r\n",
      "    │   ├── score0.951326_epoch6_time20191115160724\r\n",
      "    │   ├── score0.955394_epoch7_time20191115161808\r\n",
      "    │   ├── score0.956432_epoch10_time20191115165014\r\n",
      "    │   ├── score0.956660_epoch12_time20191115171136\r\n",
      "    │   ├── score0.957511_epoch13_time20191115172216\r\n",
      "    │   └── score0.957539_epoch15_time20191115174335\r\n",
      "    ├── \u001b[01;34mversion3\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.916125_epoch1_time20191115175722\r\n",
      "    │   ├── score0.934901_epoch2_time20191115180808\r\n",
      "    │   ├── score0.939683_epoch3_time20191115181853\r\n",
      "    │   ├── score0.945166_epoch4_time20191115182936\r\n",
      "    │   ├── score0.946322_epoch5_time20191115184019\r\n",
      "    │   ├── score0.947342_epoch6_time20191115185102\r\n",
      "    │   ├── score0.947905_epoch7_time20191115190144\r\n",
      "    │   ├── score0.950549_epoch8_time20191115191224\r\n",
      "    │   ├── score0.953166_epoch9_time20191115192305\r\n",
      "    │   └── score0.954355_epoch12_time20191115195505\r\n",
      "    ├── \u001b[01;34mversion4\u001b[00m\r\n",
      "    │   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   │   └── raw_rs.csv\r\n",
      "    │   ├── score0.928969_epoch1_time20191115204105\r\n",
      "    │   ├── score0.945035_epoch2_time20191115205152\r\n",
      "    │   ├── score0.947728_epoch3_time20191115210236\r\n",
      "    │   ├── score0.952182_epoch4_time20191115211320\r\n",
      "    │   ├── score0.952547_epoch5_time20191115212402\r\n",
      "    │   ├── score0.957946_epoch6_time20191115213444\r\n",
      "    │   ├── score0.958981_epoch10_time20191115221727\r\n",
      "    │   └── score0.961139_epoch11_time20191115222806\r\n",
      "    └── \u001b[01;34mversion5\u001b[00m\r\n",
      "        ├── \u001b[01;34mdata\u001b[00m\r\n",
      "        │   └── raw_rs.csv\r\n",
      "        ├── score0.911120_epoch1_time20191115232448\r\n",
      "        ├── score0.930853_epoch2_time20191115233538\r\n",
      "        ├── score0.932747_epoch3_time20191115234626\r\n",
      "        ├── score0.946900_epoch5_time20191116000804\r\n",
      "        ├── score0.948592_epoch7_time20191116002936\r\n",
      "        ├── score0.952430_epoch10_time20191116010156\r\n",
      "        ├── score0.955241_epoch11_time20191116011242\r\n",
      "        ├── score0.955925_epoch12_time20191116012330\r\n",
      "        └── score0.958125_epoch15_time20191116015549\r\n",
      "\r\n",
      "418 directories, 589 files\r\n"
     ]
    }
   ],
   "source": [
    "!tree $MODEL_ROOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = [pd.read_csv(p) for p in glob(join(MODEL_ROOT,'senti_entity_inplace_full_v1','*','*','raw_rs.csv')) ] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#分别统计句子的情感预测和entity的情感预测\n",
    "senti_counter = {}\n",
    "entity_counter = {}\n",
    "for df in dfs:\n",
    "    for id,negative,predicts,entity_list in df.values:\n",
    "        predicts = eval(predicts)\n",
    "        entity_list = eval(entity_list)\n",
    "        if id not in senti_counter:\n",
    "            senti_counter[id] = Counter()\n",
    "        senti_counter[id].update([negative])\n",
    "        for predict,entity in zip(predicts,entity_list):\n",
    "            key = (id,entity)\n",
    "            if key not in entity_counter:\n",
    "                entity_counter[key] = Counter()\n",
    "            entity_counter[key].update([predict])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_dict = {}\n",
    "for id,count in senti_counter.items():\n",
    "    senti = count.most_common(1)[0][0]\n",
    "    rs_dict[id] = [senti,[]]\n",
    "for (id,entity),count in entity_counter.items():\n",
    "    senti = count.most_common(1)[0][0]\n",
    "    if senti == 1:\n",
    "        rs_dict[id][1].append(entity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "items = []\n",
    "for id,(senti,entity_list) in rs_dict.items():\n",
    "    if len(entity_list) == 0:\n",
    "        entity_str = np.nan\n",
    "    else:\n",
    "        entity_str = ';'.join(entity_list)\n",
    "    items.append((id,senti,entity_str))\n",
    "df = pd.DataFrame(items,columns=['id','negative','key_entity'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('round2/tmp/inplcace_single_test.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
